# -*- coding: utf-8 -*-
import sys
import scrapy
import json
import time
import re
import datetime
from lxml import etree
from weibo_scrapy.items import MblogItem, UserItem
from weibo_scrapy.settings import LIMIT_DATE,COOKIES_ENABLED,COOKIES

HEADERS = {
    'User-Agent': 'Mozilla/5.0 (iPhone; CPU iPhone OS 11_0 like Mac OS X) AppleWebKit/604.1.38 (KHTML, like Gecko) Version/11.0 Mobile/15A372 Safari/604.1',
}

class CwbSpider(scrapy.Spider):
    name = 'cwb'
    allowed_domains = ['m.weibo.cn']
    # start_urls = ['http://m.weibo.cn/']

    def start_requests(self):
        mid = '3424883176420210'
        url = 'https://m.weibo.cn/comments/hotflow?id={mid}&mid={mid}&max_id_type=0'.format(mid=mid)
        HEADERS.update({
                'Referer': 'https://m.weibo.cn/detail/{mid}'.format(mid=mid),
            })
        yield scrapy.Request(url=url, headers=HEADERS, cookies=COOKIES,meta={'root_mid':mid})

    def parse(self, response):
        print(response.url)
        root_mid = response.meta['root_mid']
        response_data = json.loads(response.text)
        data = response_data.get('data', '')
        ok = response_data.get('ok', '')
        errno = response_data.get('errno','')
        # if not ok and errno=="100005":
        #     print('403')
        if not ok or not data:
            # print('******************', ok)
            return
        # print(data)
        # return
        if data['data']:
            for card in data['data']:
                mblog,user = card_deal(card)
                mblog['root_mid'] = root_mid
                if mblog['created_at'] < LIMIT_DATE:
                    print('*********************', mblog['created_at'])
                    return
                yield MblogItem(mblog)
                yield UserItem(user)
        max_id = data.get('max_id', '')
        max_id_type = data.get('max_id_type', '')
        # print('******************', cardlistInfo)
        if max_id:
            url = 'https://m.weibo.cn/comments/hotflow?id={mid}&mid={mid}&max_id_type={max_id_type}&max_id={max_id}'.format(mid=root_mid,max_id_type=max_id_type,max_id=max_id)
            yield scrapy.Request(url=url, cookies=COOKIES,headers=HEADERS, callback=self.parse,meta={'root_mid':root_mid})

def card_deal(card):
    mblog = card
    mblog_item = mblog_deal(mblog)
    user = mblog.get('user', '')
    user_item = user_deal(user)
    mblog_item['uid'] = user_item['uid']
    mblog_item['screen_name'] = user_item['screen_name']
    crawl_date = ts2date(time.time())
    mblog_item['crawl_date'] = user_item['crawl_date'] = crawl_date[:10]
    return mblog_item,user_item

def mblog_deal(mblog):
    mid = mblog['mid']
    bid = mblog['bid']
    comments_count = mblog.get('total_number',0)
    attitudes_count = mblog['like_count']
    created_at, timestamp = get_datetime(mblog['created_at'])
    tree = etree.HTML(mblog['text'])
    text = ''.join(tree.xpath('.//text() | .//img/@alt'))
    item = {
        'mid': mid,
        'comments_count': comments_count,
        'attitudes_count': attitudes_count,
        'created_at':created_at[:10],
        'timestamp':timestamp,
        'text':text,
            }
    return item

def user_deal(user):
    item = {}
    item['uid'] = user['id']
    # 用户昵称
    item['screen_name'] = user['screen_name']
    # 用户主页
    item['profile_url'] = user['profile_url']
    # 用户描述
    item['description'] = user['description']
    # 用户关注数
    item['follow_count'] = user['follow_count']
    # 用户粉丝数
    item['followers_count'] = user['followers_count']
    # 发帖数
    item['statuses_count'] = user['statuses_count']
    # 用户性别
    item['gender'] = user['gender']
    # 用户认证信息
    item['verified'] = user['verified']
    item['verified_type'] = user['verified_type']
    item['verified_reason'] = user.get('verified_reason', '')
    return item
def date2ts(date):
    try:
        return int(time.mktime(time.strptime(date, '%Y-%m-%d')))
    except:
        return int(time.mktime(time.strptime(date, '%Y-%m-%d %H:%M:%S')))
def ts2date(ts):
    return time.strftime('%Y-%m-%d %H:%M:%S', time.localtime(ts))
def get_datetime(date):
    format_time = time.strptime(date, '%a %b %d %H:%M:%S +0800 %Y')
    date_time = time.strftime('%Y-%m-%d %H:%M:%S', format_time)
    timestamp = int(time.mktime(format_time))
    return date_time, timestamp


# if __name__ == '__main__':
#     print trans_date_ts('刚刚')
